resolvers.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. import collections
  2. from .compat import collections_abc
  3. from .providers import AbstractResolver
  4. from .structs import DirectedGraph
  5. RequirementInformation = collections.namedtuple(
  6. "RequirementInformation", ["requirement", "parent"]
  7. )
  8. class ResolverException(Exception):
  9. """A base class for all exceptions raised by this module.
  10. Exceptions derived by this class should all be handled in this module. Any
  11. bubbling pass the resolver should be treated as a bug.
  12. """
  13. class RequirementsConflicted(ResolverException):
  14. def __init__(self, criterion):
  15. super(RequirementsConflicted, self).__init__(criterion)
  16. self.criterion = criterion
  17. def __str__(self):
  18. return "Requirements conflict: {}".format(
  19. ", ".join(repr(r) for r in self.criterion.iter_requirement()),
  20. )
  21. class InconsistentCandidate(ResolverException):
  22. def __init__(self, candidate, criterion):
  23. super(InconsistentCandidate, self).__init__(candidate, criterion)
  24. self.candidate = candidate
  25. self.criterion = criterion
  26. def __str__(self):
  27. return "Provided candidate {!r} does not satisfy {}".format(
  28. self.candidate,
  29. ", ".join(repr(r) for r in self.criterion.iter_requirement()),
  30. )
  31. class Criterion(object):
  32. """Representation of possible resolution results of a package.
  33. This holds three attributes:
  34. * `information` is a collection of `RequirementInformation` pairs.
  35. Each pair is a requirement contributing to this criterion, and the
  36. candidate that provides the requirement.
  37. * `incompatibilities` is a collection of all known not-to-work candidates
  38. to exclude from consideration.
  39. * `candidates` is a collection containing all possible candidates deducted
  40. from the union of contributing requirements and known incompatibilities.
  41. It should never be empty, except when the criterion is an attribute of a
  42. raised `RequirementsConflicted` (in which case it is always empty).
  43. .. note::
  44. This class is intended to be externally immutable. **Do not** mutate
  45. any of its attribute containers.
  46. """
  47. def __init__(self, candidates, information, incompatibilities):
  48. self.candidates = candidates
  49. self.information = information
  50. self.incompatibilities = incompatibilities
  51. def __repr__(self):
  52. requirements = ", ".join(
  53. "({!r}, via={!r})".format(req, parent)
  54. for req, parent in self.information
  55. )
  56. return "Criterion({})".format(requirements)
  57. @classmethod
  58. def from_requirement(cls, provider, requirement, parent):
  59. """Build an instance from a requirement.
  60. """
  61. candidates = provider.find_matches([requirement])
  62. if not isinstance(candidates, collections_abc.Sequence):
  63. candidates = list(candidates)
  64. criterion = cls(
  65. candidates=candidates,
  66. information=[RequirementInformation(requirement, parent)],
  67. incompatibilities=[],
  68. )
  69. if not candidates:
  70. raise RequirementsConflicted(criterion)
  71. return criterion
  72. def iter_requirement(self):
  73. return (i.requirement for i in self.information)
  74. def iter_parent(self):
  75. return (i.parent for i in self.information)
  76. def merged_with(self, provider, requirement, parent):
  77. """Build a new instance from this and a new requirement.
  78. """
  79. infos = list(self.information)
  80. infos.append(RequirementInformation(requirement, parent))
  81. candidates = provider.find_matches([r for r, _ in infos])
  82. if not isinstance(candidates, collections_abc.Sequence):
  83. candidates = list(candidates)
  84. criterion = type(self)(candidates, infos, list(self.incompatibilities))
  85. if not candidates:
  86. raise RequirementsConflicted(criterion)
  87. return criterion
  88. def excluded_of(self, candidate):
  89. """Build a new instance from this, but excluding specified candidate.
  90. Returns the new instance, or None if we still have no valid candidates.
  91. """
  92. incompats = list(self.incompatibilities)
  93. incompats.append(candidate)
  94. candidates = [c for c in self.candidates if c != candidate]
  95. if not candidates:
  96. return None
  97. criterion = type(self)(candidates, list(self.information), incompats)
  98. return criterion
  99. class ResolutionError(ResolverException):
  100. pass
  101. class ResolutionImpossible(ResolutionError):
  102. def __init__(self, causes):
  103. super(ResolutionImpossible, self).__init__(causes)
  104. # causes is a list of RequirementInformation objects
  105. self.causes = causes
  106. class ResolutionTooDeep(ResolutionError):
  107. def __init__(self, round_count):
  108. super(ResolutionTooDeep, self).__init__(round_count)
  109. self.round_count = round_count
  110. # Resolution state in a round.
  111. State = collections.namedtuple("State", "mapping criteria")
  112. class Resolution(object):
  113. """Stateful resolution object.
  114. This is designed as a one-off object that holds information to kick start
  115. the resolution process, and holds the results afterwards.
  116. """
  117. def __init__(self, provider, reporter):
  118. self._p = provider
  119. self._r = reporter
  120. self._states = []
  121. @property
  122. def state(self):
  123. try:
  124. return self._states[-1]
  125. except IndexError:
  126. raise AttributeError("state")
  127. def _push_new_state(self):
  128. """Push a new state into history.
  129. This new state will be used to hold resolution results of the next
  130. coming round.
  131. """
  132. try:
  133. base = self._states[-1]
  134. except IndexError:
  135. state = State(mapping=collections.OrderedDict(), criteria={})
  136. else:
  137. state = State(
  138. mapping=base.mapping.copy(), criteria=base.criteria.copy(),
  139. )
  140. self._states.append(state)
  141. def _merge_into_criterion(self, requirement, parent):
  142. self._r.adding_requirement(requirement, parent)
  143. name = self._p.identify(requirement)
  144. try:
  145. crit = self.state.criteria[name]
  146. except KeyError:
  147. crit = Criterion.from_requirement(self._p, requirement, parent)
  148. else:
  149. crit = crit.merged_with(self._p, requirement, parent)
  150. return name, crit
  151. def _get_criterion_item_preference(self, item):
  152. name, criterion = item
  153. try:
  154. pinned = self.state.mapping[name]
  155. except KeyError:
  156. pinned = None
  157. return self._p.get_preference(
  158. pinned, criterion.candidates, criterion.information,
  159. )
  160. def _is_current_pin_satisfying(self, name, criterion):
  161. try:
  162. current_pin = self.state.mapping[name]
  163. except KeyError:
  164. return False
  165. return all(
  166. self._p.is_satisfied_by(r, current_pin)
  167. for r in criterion.iter_requirement()
  168. )
  169. def _get_criteria_to_update(self, candidate):
  170. criteria = {}
  171. for r in self._p.get_dependencies(candidate):
  172. name, crit = self._merge_into_criterion(r, parent=candidate)
  173. criteria[name] = crit
  174. return criteria
  175. def _attempt_to_pin_criterion(self, name, criterion):
  176. causes = []
  177. for candidate in criterion.candidates:
  178. try:
  179. criteria = self._get_criteria_to_update(candidate)
  180. except RequirementsConflicted as e:
  181. causes.append(e.criterion)
  182. continue
  183. # Check the newly-pinned candidate actually works. This should
  184. # always pass under normal circumstances, but in the case of a
  185. # faulty provider, we will raise an error to notify the implementer
  186. # to fix find_matches() and/or is_satisfied_by().
  187. satisfied = all(
  188. self._p.is_satisfied_by(r, candidate)
  189. for r in criterion.iter_requirement()
  190. )
  191. if not satisfied:
  192. raise InconsistentCandidate(candidate, criterion)
  193. # Put newly-pinned candidate at the end. This is essential because
  194. # backtracking looks at this mapping to get the last pin.
  195. self._r.pinning(candidate)
  196. self.state.mapping.pop(name, None)
  197. self.state.mapping[name] = candidate
  198. self.state.criteria.update(criteria)
  199. return []
  200. # All candidates tried, nothing works. This criterion is a dead
  201. # end, signal for backtracking.
  202. return causes
  203. def _backtrack(self):
  204. # Drop the current state, it's known not to work.
  205. del self._states[-1]
  206. # We need at least 2 states here:
  207. # (a) One to backtrack to.
  208. # (b) One to restore state (a) to its state prior to candidate-pinning,
  209. # so we can pin another one instead.
  210. while len(self._states) >= 2:
  211. # Retract the last candidate pin.
  212. prev_state = self._states.pop()
  213. try:
  214. name, candidate = prev_state.mapping.popitem()
  215. except KeyError:
  216. continue
  217. self._r.backtracking(candidate)
  218. # Create a new state to work on, with the newly known not-working
  219. # candidate excluded.
  220. self._push_new_state()
  221. # Mark the retracted candidate as incompatible.
  222. criterion = self.state.criteria[name].excluded_of(candidate)
  223. if criterion is None:
  224. # This state still does not work. Try the still previous state.
  225. del self._states[-1]
  226. continue
  227. self.state.criteria[name] = criterion
  228. return True
  229. return False
  230. def resolve(self, requirements, max_rounds):
  231. if self._states:
  232. raise RuntimeError("already resolved")
  233. self._push_new_state()
  234. for r in requirements:
  235. try:
  236. name, crit = self._merge_into_criterion(r, parent=None)
  237. except RequirementsConflicted as e:
  238. raise ResolutionImpossible(e.criterion.information)
  239. self.state.criteria[name] = crit
  240. self._r.starting()
  241. for round_index in range(max_rounds):
  242. self._r.starting_round(round_index)
  243. self._push_new_state()
  244. curr = self.state
  245. unsatisfied_criterion_items = [
  246. item
  247. for item in self.state.criteria.items()
  248. if not self._is_current_pin_satisfying(*item)
  249. ]
  250. # All criteria are accounted for. Nothing more to pin, we are done!
  251. if not unsatisfied_criterion_items:
  252. del self._states[-1]
  253. self._r.ending(curr)
  254. return self.state
  255. # Choose the most preferred unpinned criterion to try.
  256. name, criterion = min(
  257. unsatisfied_criterion_items,
  258. key=self._get_criterion_item_preference,
  259. )
  260. failure_causes = self._attempt_to_pin_criterion(name, criterion)
  261. # Backtrack if pinning fails.
  262. if failure_causes:
  263. result = self._backtrack()
  264. if not result:
  265. causes = [
  266. i for crit in failure_causes for i in crit.information
  267. ]
  268. raise ResolutionImpossible(causes)
  269. self._r.ending_round(round_index, curr)
  270. raise ResolutionTooDeep(max_rounds)
  271. def _has_route_to_root(criteria, key, all_keys, connected):
  272. if key in connected:
  273. return True
  274. if key not in criteria:
  275. return False
  276. for p in criteria[key].iter_parent():
  277. try:
  278. pkey = all_keys[id(p)]
  279. except KeyError:
  280. continue
  281. if pkey in connected:
  282. connected.add(key)
  283. return True
  284. if _has_route_to_root(criteria, pkey, all_keys, connected):
  285. connected.add(key)
  286. return True
  287. return False
  288. Result = collections.namedtuple("Result", "mapping graph criteria")
  289. def _build_result(state):
  290. mapping = state.mapping
  291. all_keys = {id(v): k for k, v in mapping.items()}
  292. all_keys[id(None)] = None
  293. graph = DirectedGraph()
  294. graph.add(None) # Sentinel as root dependencies' parent.
  295. connected = {None}
  296. for key, criterion in state.criteria.items():
  297. if not _has_route_to_root(state.criteria, key, all_keys, connected):
  298. continue
  299. if key not in graph:
  300. graph.add(key)
  301. for p in criterion.iter_parent():
  302. try:
  303. pkey = all_keys[id(p)]
  304. except KeyError:
  305. continue
  306. if pkey not in graph:
  307. graph.add(pkey)
  308. graph.connect(pkey, key)
  309. return Result(
  310. mapping={k: v for k, v in mapping.items() if k in connected},
  311. graph=graph,
  312. criteria=state.criteria,
  313. )
  314. class Resolver(AbstractResolver):
  315. """The thing that performs the actual resolution work.
  316. """
  317. base_exception = ResolverException
  318. def resolve(self, requirements, max_rounds=100):
  319. """Take a collection of constraints, spit out the resolution result.
  320. The return value is a representation to the final resolution result. It
  321. is a tuple subclass with three public members:
  322. * `mapping`: A dict of resolved candidates. Each key is an identifier
  323. of a requirement (as returned by the provider's `identify` method),
  324. and the value is the resolved candidate.
  325. * `graph`: A `DirectedGraph` instance representing the dependency tree.
  326. The vertices are keys of `mapping`, and each edge represents *why*
  327. a particular package is included. A special vertex `None` is
  328. included to represent parents of user-supplied requirements.
  329. * `criteria`: A dict of "criteria" that hold detailed information on
  330. how edges in the graph are derived. Each key is an identifier of a
  331. requirement, and the value is a `Criterion` instance.
  332. The following exceptions may be raised if a resolution cannot be found:
  333. * `ResolutionImpossible`: A resolution cannot be found for the given
  334. combination of requirements. The `causes` attribute of the
  335. exception is a list of (requirement, parent), giving the
  336. requirements that could not be satisfied.
  337. * `ResolutionTooDeep`: The dependency tree is too deeply nested and
  338. the resolver gave up. This is usually caused by a circular
  339. dependency, but you can try to resolve this by increasing the
  340. `max_rounds` argument.
  341. """
  342. resolution = Resolution(self.provider, self.reporter)
  343. state = resolution.resolve(requirements, max_rounds=max_rounds)
  344. return _build_result(state)